
#-*- coding:utf-8 -*- 
import codecs 
from whoosh.index import create_in  
from whoosh.fields import *  
from chinesetokenizer import ChineseAnalyzer
#from whoosh.analysis import RegexAnalyzer  
#analyzer = RegexAnalyzer(ur"([\u4e00-\u9fa5])|(\w+(\.?\w+)*)")
import random
from whoosh.index import open_dir
from whoosh import qparser
from whoosh.collectors import TimeLimitCollector, TimeLimit

def cmp(t1, t2):
    k1, v1  = t1;
    k2, v2 = t2;
    return len(v2) - len(v1);

def getScore(total, pos):
    bar = total/3;
    if pos < bar:
        return 3;
    elif pos < bar*2:
        return 2;
    else:
        return 1;
def computeDis(results, value):
    if results == None:
        print 'Wrong';
        return 0;
    setA = set([hit.fields()['appid'] for hit in results]);
    setB = set(value.keys());
    l = len(setA.intersection(setB));

    return l;

def parseQD(s):
    sl = s.split('|');
    di = {};
    for i in range(1, len(sl)):
        id, appID, fre =  sl[i].split(':');
        di[appID] = fre;
    return (sl[0], di);

ix = open_dir("IndexDir/titleIndex");
parser = qparser.QueryParser('content',ix.schema, group=qparser.syntax.OrGroup);
searcher = ix.searcher()   ;
c = searcher.collector(limit=10);
tlc = TimeLimitCollector(c, timelimit=30);
with open('test_collelctions', 'w') as filter:
    logstatistical = open('bm25_unmatch_statistical', 'r');
    count = 0;
    for line in logstatistical:
        li = line.strip().split('\t', 1);
        score = float(li[0]);
        if score == 0.0 and random.random() < 0.01:
            query, value = parseQD(li[1]);
            l = len(value);
            formatValue = map(lambda x: '%d:%s:%d' % (x[0], x[1][0], getScore(l, x[0])),enumerate(value.items(),start=1));   
            filter.write('%s|%s\n' %(query, '|'.join(formatValue)));
    logstatistical.close();
